# -*- coding: utf-8 -*-
"""
Created on Wed Feb 03 17:41:48 2016

@author: Administrator
"""
import json
import re
from bs4 import BeautifulSoup
from six import unichr

options_list = [  # list("①②③④"),
    ["(Ⅰ)", "(Ⅱ)", "(Ⅲ)", "(Ⅳ)"],
    ["(i)", "(ii)", "(iii)"],
    ["(I)", "(II)", "(III)"],
    ["Ⅰ", "Ⅱ", "Ⅲ", "Ⅳ"], ["⑴", "⑵"], ["( I )", "( II )", "( III )"],
    ["I", "( II )", "( III )"], ["I.", "II.", "III."],
    ["（1）", "（2）", "（3）"],
    ["(1)", "(2)", "(3)", "(4)", "(5)"]]


def q2b(uchar):
    """全角转半角"""
    if ord(uchar) == 20008:
        return "|"

    inside_code = ord(uchar)
    if inside_code == 0x3000:
        inside_code = 0x0020
    else:
        inside_code -= 0xfee0
    if inside_code < 0x0020 or inside_code > 0x7e:  # 转完之后不是半角字符返回原来的字符
        return uchar
    return unichr(inside_code)


def string_q2b(ustring):
    """把字符串全角转半角"""
    return "".join([q2b(uchar) for uchar in ustring])


def clean_text(text):
    text = text.replace(",,", ",").replace("\xa0", "").replace('\ue00a', "").replace("\uf04f", "").replace('\ue009', "").replace("\u200a", "").replace("\u200b", "").replace("\u200c", "").replace("\u200d", "").replace("∣", "|")
    text = text.replace("_{}", "").replace("{)}^", ")^").replace("{}^", "^").replace("{}_", "").replace("^{\\circ}", "°")
    text = text.replace("{}", "").replace("{log}", "log").replace("{∪}", "{U}").replace("{u}", "{U}").replace("\t", "").replace("\n", "")
    text = text.replace("×", "*").replace("÷", "/").replace('\uef05', ",").replace("\u2500", "-")
    text = text.replace('\ufe59', "(").replace('\ufe5a', ")").replace('\uff09', ")").replace('\uff08', "(").replace("...", "…")
    text = text.replace("\\lbrack", "[").replace("\\rbrack", "]").replace("\\lbrace", "{").replace("\\rbrace", "}").replace("\\lvert", "|").replace("\\rvert", "|")
    text = text.replace("\\alpha", 'α').replace("\\cdots", "…").replace("\\cdot", "\u2022").replace("\\bar ", "\\overline ")
    text = text.replace("\\geqslant ", "\u2265").replace("\\leqslant ", "\u2264").replace("\\pi", "π").replace("\\phi", "φ")
    text = text.replace("\\left", "").replace("\\right.", "").replace("\\right", "").replace("\\pm ", "±").replace("*s ", "…").replace("\\infty", '\u221e')
    text = text.replace("\\lambda", 'λ').replace("\\omega", 'ω').replace("\\beta", 'β').replace("\\mu ", 'μ').replace("\\varphi", '\u03b2').replace("\\rho ", 'ρ')
    text = text.replace("\\gamma", 'γ').replace("\\delta", '\u03c9').replace("\\epsilon", 'ε').replace("\\tau ", '\u03b2').replace("\\zeta ", '\u03b2').replace("\\theta ", 'θ')
    text = text.replace("\\xi ", 'ξ').replace("\\iota ", '\u03c9').replace("\\kappa", '\u03b2').replace("\\nu ", '\u03b2').replace("\\eta ", '\u03b2').replace("\\sigma", 'σ')
    text = text.replace("\\varepsilon", '\u03bb').replace("\\vartheta", '\u03c9').replace("\\varpi", '\u03b2').replace("\\varrho", '\u03b2')
    text = text.replace("\\Gamma", 'Γ').replace("\\Delta", '\u03c9').replace("\\Theta", '\u03b2').replace("\\Lambda", '\u03b2')
    text = text.replace("\\Xi ", '\u03bb').replace("\\Pi ", '\u03c9').replace("\\Sigma", '\u03b2').replace("\\Upsilon", '\u03b2')
    text = text.replace("\\Phi ", 'Φ').replace("\\Psi ", '\u03c9').replace("\\Omega", '\u03b2')
    text = text.replace("\\hat ", '\\widehat ').replace("\\subset ", "⊆").replace("\\subseteq ", "⊆").replace("\\neq ", '\u2260').replace("\\mid ", "|").replace("\\simeq", "\u224C").replace("\\sim ", "\u223d").replace("\\approx ", "\u223d")
    text = text.replace("\\qquad", "").replace("\\pm ", "\u00B1").replace("\\displaystyle ", "").replace("\\in ", '\u2208')
    text = text.replace("\\cap", "∩").replace("\\cup", "\u222a").replace("\\mathrm", "").replace("\\mathbf", "").replace('\ufe63', "-").replace("\\perp ", '\u22a5').replace("\\parallel", '\u2225').replace("\\bot ", '\u22a5')
    text = text.replace("\\complement_", "C_{U}").replace("\\complement _", "C_{U}").replace("\\times", "*")
    text = text.replace("\\int ", "\u222b").replace("\\odot ", "⊙").replace("\\%", "").replace("\\vartriangle", "\\triangle")
    text = text.replace("\\forall", "∀").replace("\\exists", "∃").replace("\\notin", "\u2209").replace("\\varnothing", "∅")
    text = text.replace('\ufe5b', "{").replace('\ufe5c', "}")
    text = re.sub("选修.*不等式选讲", "", text)
    text = re.sub("[_]{2,}", "", text)
    text = re.sub("^\\(20[0-9]{2}.*\\)", "", text)
    return text


def get_subs(text):
    main_text = text
    for opt_num, options in enumerate(options_list):
        is_multi = 0
        sub_text = []
        for i, option_flag in enumerate(options, 1):
            if option_flag in main_text:
                if opt_num != len(options_list) - 1:
                    items = main_text.split(option_flag)
                else:
                    res = re.match("\(%d\)(.*)" % i, main_text)
                    if res:
                        items = ["", res.group(1)]
                    else:
                        items = re.findall("(.*[^fg])\(%d\)(.*)" % i, main_text)
                        items = [token for item in items for token in item]

                if len(items) == 0:
                    return [text]
                elif len(items) == 1:
                    return [text]
                elif len(items) > 2:
                    sub = items[0]
                    main_text = "".join(items[1:])
                else:
                    sub, main_text = items

                sub_text.append(sub)
                # print "-->:", sub
                # print "-->:", main_text

            elif i > 2:
                is_multi = 1
                break
            else:
                break

        if is_multi == 0 and len(sub_text) == len(options):
            is_multi = 1

        if len(sub_text) > 0:
            sub_text.append(main_text)

        if is_multi == 1:
            break

    if is_multi:
        return sub_text
    else:
        return [text]

        # items = get_subs("已知函数f(x)=x^{3}-ax^{2}+10,(I)当a=1时,求曲线y=f(x)在点(1,f(1))处的切线方程;(Ⅱ)在区间[4,6]内至少存在一个实数x,使得f(x)<0成立,求实数a的取值范围.(Ⅲ)当a=1时,设函数g(x)=lnx-2x^{2}+4x+t(t为常数),若使g(x)≤x+m≤f(x)+x-10在(0,+∞)上恒成立的实数m有,只有一个,求实数m和t的值.")
        # for item in items:
        # print item


# raise

symbol = ['log', 'ln', 'sin', 'cos', 'tan', 'csc', 'sec', 'cot']
symbol_rule = dict([("%s{%s}" % (sym[:-1], sym[-1]), sym) for sym in symbol])


def process(item):
    doc_id, raw_text = item
    raw_text = raw_text.replace('<label class="underline"></label>', "()")
    raw_text = raw_text.replace("&nbsp;", '')
    raw_text = raw_text.replace("@#", '').replace("#@", '')
    raw_text = raw_text.replace("<sup>", "^{").replace("</sup>", "}").replace("<sub>", "_{").replace("</sub>", "}")
    # raw_text = stringQ2B(raw_text.replace("_{}", ""))

    if "<div" in raw_text or "<p" in raw_text or '<tex' in raw_text or '<answer' in raw_text:
        soup = BeautifulSoup(raw_text, 'html.parser')
        # text = ""
        # if soup.find("p"):
        # text += soup.find("p").text
        text = string_q2b(soup.text)

        options = [string_q2b(lab.text) for lab in soup.findAll("label") if lab.text]
    else:
        text = string_q2b(raw_text.decode("utf-8"))
        options = []

    if len(options) > 0 and options != ["A.", "B.", "C.", "D."]:
        if "A." not in options[0]:
            print(options[0])

        text = text.split(options[0])[0]
        options = [re.sub("[ABCD]\\.", "", opt) for opt in options]
    elif all([op in text for op in ["A:", "B:", "C:", "D:"]]):
        items = text.split("A:")
        if len(items) == 2:
            text, op_text = items
            options = []
            for op in ["B:", "C:", "D:"]:
                option, op_text = op_text.split(op)
                options.append(option)
            options.append(op_text)
    elif all([op in text for op in ["A、", "B、", "C、", "D、"]]):
        items = text.split("A、")
        if len(items) == 2:
            text, op_text = items
            options = []
            for op in ["B、", "C、", "D、"]:
                option, op_text = op_text.split(op)
                options.append(option)
            options.append(op_text)
    elif all([op in text for op in ["A.", "B.", "C.", "D."]]):
        items = text.split("A.")
        if len(items) == 2:
            text, op_text = items
            options = []
            for op in ["B.", "C.", "D."]:
                option, op_text = op_text.split(op)
                options.append(option)
            options.append(op_text)

    options = [clean_text(opt) for opt in options]

    for wrong, correct in symbol_rule.items():
        text = text.replace(wrong, correct)

    # for sym in list("①②③④⑤⑥⑦⑧"):
    #     text = text.replace(sym, "")

    text = clean_text(text)
    subs = get_subs(text)
    return str(doc_id) + "\t" + json.dumps({"main": subs[0], "options": options, "subquestions": subs[1:]},
                                           ensure_ascii=False).encode("utf-8") + "\n"


def preprocess(raw_text):
    raw_text = raw_text.replace('<label class="underline"></label>', "()")
    raw_text = raw_text.replace("&nbsp;", '')
    raw_text = raw_text.replace("@#", '').replace("#@", '')
    raw_text = raw_text.replace("<sup>", "^{").replace("</sup>", "}").replace("<sub>", "_{").replace("</sub>", "}")
    imgs = []
    if "<div" in raw_text or "<p" in raw_text or '<tex' in raw_text or '<answer' in raw_text or '<img' in raw_text or '<span':
        soup = BeautifulSoup(raw_text, 'html.parser')
        for img in soup.find_all("img"):
            imgs.append(img.get("src"))
        text = string_q2b(soup.text)
    else:
        if isinstance(raw_text, str):
            raw_text = raw_text.decode("utf-8")
        text = string_q2b(raw_text)

    for wrong, correct in symbol_rule.items():
        text = text.replace(wrong, correct)
    return text, imgs


def process_new(doc_id, raw_text, options):
    text, main_img = preprocess(raw_text)
    option_img = []
    if options:
        js = json.loads(options)
        options = [js[name] for name in "ABCD"]
        new_options = []
        for opt in options:
            option, imgs = preprocess(opt)
            new_options.append(option)
            option_img.extend(imgs)

        options = new_options
    else:
        options = []

    options = [clean_text(opt) for opt in options]

    # for sym in list("①②③④⑤⑥⑦⑧"):
    #     text = text.replace(sym, "")

    text = clean_text(text)
    subs = get_subs(text)
    return str(doc_id) + "\t" + json.dumps(
        {"main": subs[0], "options": options, "subquestions": subs[1:], "main_img": main_img, "option_img": option_img},
        ensure_ascii=False).encode("utf-8") + "\n"


if __name__ == '__main__':
    pass
